###############################################
##     OVAL Content management library       ##
## By Kirill Ermakov @ Positive Technologies ##
##                                           ##
## www.ptsecurity.com www.maxpatrol.com      ##
###############################################

import sys
reload(sys)
sys.setdefaultencoding('utf-8')
###### Magic =)
import collections
import functools
from itertools import ifilterfalse
from heapq import nsmallest
from operator import itemgetter
from ZODB.FileStorage import FileStorage
from ZODB.DB import DB
import re
import persistent
from persistent.mapping import PersistentMapping
from persistent.list import PersistentList
import xml.etree.cElementTree as etree
import datetime
import os
from oval_player import oval_mapper
import hashlib

# Counter class
class Counter(dict):
    def __missing__(self, key):
        return 0
# Hashable dict
class FrozenDict(collections.Mapping):
    """Don't forget the docstrings!!"""

    def __init__(self, *args, **kwargs):
        self._d = dict(*args, **kwargs)

    def __iter__(self):
        return iter(self._d)

    def __len__(self):
        return len(self._d)

    def __getitem__(self, key):
        return self._d[key]

    def __hash__(self):
        return hash(tuple(sorted(self._d.iteritems())))
# LFU Caching function
def lfu_cache(maxsize=100):

    def decorating_function(user_function):
        cache = {}                      # mapping of args to results
        use_count = Counter()           # times each key has been accessed
        kwd_mark = object()             # separate positional and keyword args

        @functools.wraps(user_function)
        def wrapper(*args, **kwds):
            key = args
            if kwds:
                key += (kwd_mark,) + tuple(sorted(kwds.items()))
            use_count[key] += 1

            # get cache entry or compute if not found
            try:
                result = cache[key]
                wrapper.hits += 1
            except KeyError:
                result = user_function(*args, **kwds)
                cache[key] = result
                wrapper.misses += 1

                # purge least frequently used cache entry
                if len(cache) > maxsize:
                    for key, _ in nsmallest(maxsize // 10,
                                            use_count.iteritems(),
                                            key=itemgetter(1)):
                        del cache[key], use_count[key]

            return result

        def clear():
            cache.clear()
            use_count.clear()
            wrapper.hits = wrapper.misses = 0

        wrapper.hits = wrapper.misses = 0
        wrapper.clear = clear
        return wrapper
    return decorating_function
# MAIN DATABASE CONNECTOR CLASS
class dbconnector():
    def __init__(self, filename):
        self.storage = None
        self.db = None
        self.root = None
        self.connection = None
        if filename:
            self.storage = FileStorage(filename)
            self.db = DB(self.storage)
            self.connection = self.db.open()
            self.root = self.connection.root()
            self.transaction = self.connection.transaction_manager.begin()
    def commit(self):
        self.transaction.commit()
        self.transaction = self.connection.transaction_manager.begin()
# UNIVERSAL VARIABLE WITH SELF HASH FUNCTION
class variable(persistent.Persistent):
    def __init__(self, tag=str(), body = str(), attributes = None, variables = None):
        if not attributes:
            self.attributes = PersistentMapping()
        if not variables:
            self.variables = PersistentList()
        if attributes and not isinstance(attributes, PersistentMapping):
            attributes = PersistentMapping(attributes)
        if variables and not isinstance(variables, PersistentList):
            variables = PersistentList(variables)
        self.tag = tag
        self.body = body
        if attributes:
            self.attributes = attributes
        if variables:
            self.variables = variables
        self.vHash = 0

    def clearme(self):
        hashmap = PersistentMapping()
        varlist = list(self.variables)
        for var in varlist:
            var.clearme()
            var_hash = var.hashme()
            if var_hash not in hashmap.keys():
                hashmap[var_hash] = None
            else:
                self.variables.remove(var)

    def hashme(self):
        summ_hash = hashlib.md5()
        summ_hash.update(str(self.tag))
        summ_hash.update(str(self.body))
        if self.attributes:
            for attribute in self.attributes.keys():
                summ_hash.update(str(attribute))
                summ_hash.update(str(self.attributes[attribute]))
        if self.variables:
            for var in self.variables:
                if isinstance(var, variable):
                    if var != self:
                        summ_hash.update(var.hashme())
                else:
                    raise Exception("Error input parameters in variables. Self append ?")
        self.vHash = summ_hash.hexdigest()
        return self.vHash
# UNIVERSAL OVAL OBJECTS
class oval_object(persistent.Persistent):
    def __init__(self):
        self.id = str()
        self.tag = "object"
        self.namespace = str()
        self.oval = str()
        self.version = 0
        self.vHash = 0
        self.deprecated = False
        self.variables = PersistentList()
        self.notes = PersistentList()
        self.comment = str()
        self.signature = str()

    def assign_id(self, UniqId):
        self.id = "oval:"+self.oval+":obj:"+str(UniqId)

    def hash(self):
        summ_hash = hashlib.md5()
        summ_hash.update(str(self.tag))
        summ_hash.update(str(self.namespace))
        summ_hash.update(str(self.comment))
        
        for var in self.variables:
            if isinstance(var, variable):
                summ_hash.update(var.hashme())
            else:
                raise Exception("Error input parameters in object variables")
        return summ_hash.hexdigest()

    def hashme(self):
        self.vHash = self.hash()
        return self.vHash

    def clearme(self):
        hashmap = PersistentMapping()
        varlist = list(self.variables)
        for var in varlist:
            var.clearme()
            var_hash = var.hashme()
            if var_hash not in hashmap.keys():
                hashmap[var_hash] = None
            else:
                self.variables.remove(var)

class oval_state(oval_object):
    def __init__(self):
        super(oval_state, self).__init__()
        self.tag = "state"
        self.operator = str()
    def assign_id(self, UniqId):
        self.id = "oval:"+self.oval+":ste:"+str(UniqId)
    def hashme(self):
        summ_hash = hashlib.md5()
        summ_hash.update(self.hash())
        summ_hash.update(str(self.operator))
        self.vHash = summ_hash.hexdigest()
        return self.vHash

class oval_variable(oval_object):
    def __init__(self):
        super(oval_variable, self).__init__()
        self.tag = "variable"
        self.datatype = str()
    def assign_id(self, UniqId):
        self.id = "oval:"+self.oval+":var:"+str(UniqId)
    def hashme(self):
        summ_hash = hashlib.md5()
        summ_hash.update(self.hash())
        summ_hash.update(str(self.datatype))
        self.vHash = summ_hash.hexdigest()
        return self.vHash

class oval_test(oval_object):
    def __init__(self):
        super(oval_test, self).__init__()
        self.tag = "test"
        self.check_existance = str()
        self.check = str()
        self.state_operator = str()
    def assign_id(self, UniqId):
        self.id = "oval:"+self.oval+":tst:"+str(UniqId)
    def hashme(self):
        summ_hash = hashlib.md5()
        summ_hash.update(self.hash())
        summ_hash.update(str(self.check))
        summ_hash.update(str(self.check_existance))
        summ_hash.update(str(self.state_operator))
        self.vHash = summ_hash.hexdigest()
        return self.vHash

class oval_definition(oval_object):
    def __init__(self):
        super(oval_definition, self).__init__()
        self.tag = "definition"
        self.defclass = ""
        self.title = ""
        self.description = ""
        self.metadata = PersistentList()
        self.criteria = PersistentList()
    def assign_id(self, UniqId):
        self.id = "oval:"+self.oval+":def:"+str(UniqId)
    def hashme(self):
        summ_hash = hashlib.md5()
        # Dognail for situation, where titles are the same...
        if hasattr(self, 'hiddentitle') and self.hiddentitle and isinstance(self.hiddentitle, basestring):
            summ_hash.update(str(self.hiddentitle))
        else:
            summ_hash.update(str(self.title))
        self.vHash = summ_hash.hexdigest()
        return self.vHash
    def datahash(self):
        summ_hash = hashlib.md5()
        summ_hash.update(str(self.description))
        for element in self.metadata:
            summ_hash.update(element.hashme())
        for element in self.criteria:
            summ_hash.update(element.hashme())
        return summ_hash.hexdigest()
    def clearme(self):

        hashmap = PersistentMapping()
        varlist = list(self.variables)
        for var in varlist:
            var.clearme()
            var_hash = var.hashme()
            if var_hash not in hashmap.keys():
                hashmap[var_hash] = None
            else:
                self.variables.remove(var)

        hashmap = PersistentMapping()
        varlist = list(self.metadata)
        for var in varlist:
            var.clearme()
            var_hash = var.hashme()
            if var_hash not in hashmap.keys():
                hashmap[var_hash] = None
            else:
                self.metadata.remove(var)

        hashmap = PersistentMapping()
        varlist = list(self.criteria)
        for var in varlist:
            var.clearme()
            var_hash = var.hashme()
            if var_hash not in hashmap.keys():
                hashmap[var_hash] = None
            else:
                self.criteria.remove(var)

class oval_suite():
    def __init__(self, dbname = None, name = None, schema_version = None, product_version = None):

        self.namespace = "http://ptsecurity.ru/ptoval"
        if name:
            self.name = name
        else:
            self.name = "com.ptsecurity.default"
        if schema_version:
            self.schema_version = schema_version
        else:
            self.schema_version = "5.10"
        if product_version:
            self.product_version = product_version
        else:
            self.product_version = "1.0"
            
        if dbname and isinstance(dbname,basestring):
            self.dbname = dbname
            self.db = dbconnector(dbname)
            self._init_db()
            self._init_default_hashes()

    def _init_default_hashes(self):
        self.default_hashmap = PersistentMapping()
        new = oval_definition()
        self.default_hashmap[new.hashme()] = None
        new = oval_object()
        self.default_hashmap[new.hashme()] = None
        new = oval_state()
        self.default_hashmap[new.hashme()] = None
        new = oval_test()
        self.default_hashmap[new.hashme()] = None
        new = oval_variable()
        self.default_hashmap[new.hashme()] = None

    def _init_db(self):
        if "definitions" in self.db.root and "variables" in self.db.root and "objects" in self.db.root and "tests" in self.db.root and "states" in self.db.root and "hashmap" in self.db.root and "index" in self.db.root and "name" in self.db.root and "master" in self.db.root:
            self.master = self.db.root["master"]
            self.definitions = self.db.root["definitions"]
            self.objects = self.db.root["objects"]
            self.tests = self.db.root["tests"]
            self.states = self.db.root["states"]
            self.variables = self.db.root["variables"]
            self.hashmap = self.db.root["hashmap"]
            self.index = self.db.root["index"]
            self.name = self.db.root["name"]
        else:
            self.db.root["master"] = PersistentMapping()
            self.master = self.db.root["master"]
            self.db.root["definitions"] = PersistentMapping()
            self.definitions = self.db.root["definitions"]
            self.db.root["objects"] = PersistentMapping()
            self.objects = self.db.root["objects"]
            self.db.root["tests"] = PersistentMapping()
            self.tests = self.db.root["tests"]
            self.db.root["states"] = PersistentMapping()
            self.states = self.db.root["states"]
            self.db.root["variables"] = PersistentMapping()
            self.variables = self.db.root["variables"]
            self.db.root["hashmap"] = PersistentMapping()
            self.hashmap = self.db.root["hashmap"]
            self.db.root["namespace"] = self.namespace
            self.db.root["index"] = PersistentMapping()
            self.index = self.db.root["index"]
            self.index["definitions"] = 0
            self.index["objects"] = 0
            self.index["states"] = 0
            self.index["tests"] = 0
            self.index["variables"] = 0
            self.db.root["name"] = self.name

    def savedb(self):
        self.db.root["definitions"] = self.definitions
        self.db.root["objects"] = self.objects
        self.db.root["tests"] = self.tests
        self.db.root["states"] = self.states
        self.db.root["variables"] = self.variables
        self.db.root["hashmap"] = self.hashmap
        self.db.root["index"] = self.index
        self.db.commit()

    def close(self):
        self.savedb()
        self.db.db.pack()
        self.db.db.close()
        # Garbage remover
        if os.path.exists(self.dbname+".tmp"):
            os.unlink(self.dbname+".tmp")
        if os.path.exists(self.dbname+".index"):
            os.unlink(self.dbname+".index")
        if os.path.exists(self.dbname+".old"):
            os.unlink(self.dbname+".old")
        if os.path.exists(self.dbname+".lock"):
            os.unlink(self.dbname+".lock")

    def fclose(self):
        self.savedb()
        self.db.db.close()
        # Garbage remove
        if os.path.exists(self.dbname+".tmp"):
            os.unlink(self.dbname+".tmp")
        if os.path.exists(self.dbname+".index"):
            os.unlink(self.dbname+".index")
        if os.path.exists(self.dbname+".old"):
            os.unlink(self.dbname+".old")
        if os.path.exists(self.dbname+".lock"):
            os.unlink(self.dbname+".lock")

    def stats(self):
        statistics = PersistentMapping()
        statistics['definitions'] = len(self.definitions)
        statistics['tests'] = len(self.tests)
        statistics['objects'] = len(self.objects)
        statistics['states'] = len(self.states)
        statistics['variables'] = len(self.variables)
        statistics['hashmap'] = len(self.hashmap)
        return statistics

    def put(self, object):
        if object:
            if not object.namespace:
                if not isinstance(object, oval_definition):
                    object.namespace = self.namespace + "#" + self.name.split(".")[len(self.name.split("."))-1]
                else:
                    object.namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5"
            object.oval = self.name
            object.clearme()
            object_hash = object.hashme()
            if (object_hash not in self.default_hashmap) and (object_hash in self.hashmap.keys()):
                object.id = self.hashmap[object_hash][1]
                old_object = self.db.root[self.hashmap[object_hash][0]][self.hashmap[object_hash][1]]
                if hasattr(object, "datahash"):
                    data_hash = object.datahash()
                    if data_hash not in self.hashmap.keys():
                        if object.version <= old_object.version:
                            object.version = old_object.version + 1
                        old_data_hash = old_object.datahash()
                        self.hashmap[data_hash] = (self.hashmap[old_data_hash][0],object.id)
                        del self.hashmap[old_data_hash]
                    else:
                        object.version = old_object.version
                del old_object
                self.master[object.id] = object
                if isinstance(object, oval_state):
                    self.states[object.id] = object
                elif isinstance(object, oval_test):
                    self.tests[object.id] = object
                elif isinstance(object, oval_definition):
                    self.definitions[object.id] = object
                elif isinstance(object, oval_variable):
                    self.variables[object.id] = object
                elif isinstance(object, oval_object):
                    self.objects[object.id] = object
                else:
                    raise Exception("Illegal input argument. Only basic OVAL classes.")
            else:
                if isinstance(object, oval_state):
                    if not object.id:
                        object.assign_id(self.index["states"])
                    self.hashmap[object_hash] = ("states",object.id)
                    self.states[object.id] = object
                    self.master[object.id] = object
                    self.index["states"] += 1
                elif isinstance(object, oval_test):
                    if not object.id:
                        object.assign_id(self.index["tests"])
                    self.hashmap[object_hash] = ("tests",object.id)
                    self.tests[object.id] = object
                    self.master[object.id] = object
                    self.index["tests"] += 1
                elif isinstance(object, oval_definition):
                    if not object.id:
                        object.assign_id(self.index["definitions"])
                    self.hashmap[object_hash] = ("definitions",object.id)
                    self.hashmap[object.datahash()] = ("definitions",object.id)
                    self.definitions[object.id] = object
                    self.master[object.id] = object
                    self.index["definitions"] += 1
                elif isinstance(object, oval_variable):
                    if not object.id:
                        object.assign_id(self.index["variables"])
                    self.hashmap[object_hash] = ("variables",object.id)
                    self.variables[object.id] = object
                    self.master[object.id] = object
                    self.index["variables"] += 1
                elif isinstance(object, oval_object):
                    if not object.id:
                        object.assign_id(self.index["objects"])
                    self.hashmap[object_hash] = ("objects",object.id)
                    self.objects[object.id] = object
                    self.master[object.id] = object
                    self.index["objects"] += 1
                else:
                    raise Exception("Illegal input argument. Only basic OVAL classes.")
            return object.id
        else:
            raise Exception("Empty input object. Please, refer to the manual.")

    def xml_me(self, object, easy = 0):
        if object and (isinstance(object, oval_object) or isinstance(object, variable)):
            if isinstance(object, variable):
                var = etree.Element(object.tag)
                if object.body:
                    var.text = object.body
                for k in object.attributes.keys():
                    var.attrib[k] = object.attributes[k]
                for sub in object.variables:
                    var.append(self.xml_me(sub))
                return var
            elif isinstance(object, oval_definition):
                definition = etree.Element(object.tag)
                definition.attrib['id'] = object.id
                if object.deprecated:
                    definition.attrib['deprecated'] = "true"
                definition.attrib['class'] = object.defclass
                definition.attrib['version'] = str(object.version)
                definition.attrib['xmlns'] = object.namespace
                metadata = etree.SubElement(definition,"metadata")
                etree.SubElement(metadata, "title").text = str(object.title)
                if object.comment:
                    definition.attrib['comment'] = object.comment
                if not easy:
                    etree.SubElement(metadata, "description").text = unicode(object.description)
                    for element in object.metadata:
                        metadata.append(self.xml_me(element))
                    for element in object.notes:
                        metadata.append(self.xml_me(element))
                for element in object.criteria:
                    definition.append(self.xml_me(element))
                return definition
            elif isinstance(object, oval_test):
                test = etree.Element(object.tag)
                test.attrib['id'] = object.id
                test.attrib['version'] = str(object.version)
                test.attrib['xmlns'] = object.namespace
                if object.deprecated:
                    test.attrib['deprecated'] = "true"
                if object.comment:
                    test.attrib['comment'] = object.comment
                if object.check:
                    test.attrib['check'] = object.check
                if object.check_existance:
                    test.attrib['check_existance'] = object.check_existance
                if object.state_operator:
                    test.attrib['state_operator'] = object.state_operator
                for element in object.variables:
                    test.append(self.xml_me(element))
                for element in object.notes:
                    test.append(self.xml_me(element))
                return test
            elif isinstance(object, oval_state):
                state = etree.Element(object.tag)
                state.attrib['id'] = object.id
                state.attrib['version'] = str(object.version)
                state.attrib['xmlns'] = object.namespace
                if object.operator:
                    state.attrib['operator'] = object.operator
                if object.deprecated:
                    state.attrib['deprecated'] = "true"
                if object.comment:
                    state.attrib['comment'] = object.comment
                for element in object.variables:
                    state.append(self.xml_me(element))
                for element in object.notes:
                    state.append(self.xml_me(element))
                return state
            elif isinstance(object, oval_variable):
                var = etree.Element(object.tag)
                var.attrib['id'] = object.id
                var.attrib['version'] = str(object.version)
                var.attrib['xmlns'] = object.namespace
                if object.datatype:
                    var.attrib['datatype'] = object.datatype
                if object.deprecated:
                    var.attrib['deprecated'] = "true"
                if object.comment:
                    var.attrib['comment'] = object.comment
                for element in object.variables:
                    var.append(self.xml_me(element))
                for element in object.notes:
                    var.append(self.xml_me(element))
                return var
            elif isinstance(object, oval_object):
                obj = etree.Element(object.tag)
                obj.attrib['id'] = object.id
                obj.attrib['version'] = str(object.version)
                obj.attrib['xmlns'] = object.namespace
                if object.deprecated:
                    obj.attrib['deprecated'] = "true"
                if object.comment:
                    obj.attrib['comment'] = object.comment
                for element in object.variables:
                    obj.append(self.xml_me(element))
                for element in object.notes:
                    obj.append(self.xml_me(element))
                return obj
        else:
            raise Exception("Only OVAL objects usable as input")

    @lfu_cache(maxsize=50)
    def get(self, id):
        if id and id in self.master:
            return self.master[id]
        else:
            return None

    def delete(self, id):
        if id and id in self.master:
            object = self.master[id]
            hash = object.hashme()
            index = int(id.split(":")[len(id.split(":"))-1])
            if isinstance(object, oval_test) and id in self.tests:
                if self.index['tests'] == index + 1:
                    self.index['tests'] -= 1
                del self.tests[id]
            if isinstance(object, oval_state) and id in self.states:
                if self.index['states'] == index + 1:
                    self.index['states'] -= 1
                del self.states[id]
            if isinstance(object, oval_definition) and id in self.definitions:
                if self.index['definitions'] == index + 1:
                    self.index['definitions'] -= 1
                del self.definitions[id]
            if isinstance(object, oval_variable) and id in self.variables:
                if self.index['variables'] == index + 1:
                    self.index['variables'] -= 1
                del self.variables[id]
            if isinstance(object, oval_object) and id in self.objects:
                if self.index['objects'] == index + 1:
                    self.index['objects'] -= 1
                del self.objects[id]
            if hash in self.hashmap:
                del self.hashmap[hash]
            if hasattr(object, "datahash"):
                datahash = object.datahash()
                if datahash in self.hashmap:
                    del self.hashmap[datahash]
            del object
            del self.master[id]

    def variable_search(self, variable, pattern):
        if variable:
            summarizing_string = variable.tag
            for attr in variable.attributes:
                summarizing_string += attr
                summarizing_string += str(variable.attributes[attr])
            summarizing_string += str(variable.body)
            if summarizing_string.find(str(pattern)) != -1 or summarizing_string.find(str(pattern).lower()) != -1 or summarizing_string.find(str(pattern).upper()) != -1:
                self_value = True
            else:
                self_value = False
            for var in variable.variables:
                self_value = self_value or self.variable_search(var, pattern)
            return self_value
        else:
            return False
            
    @lfu_cache(maxsize=50)
    def search(self, data = FrozenDict()):
        # Some nice search algorithm
        if data and isinstance(data, (PersistentMapping, dict, FrozenDict)):
            result = PersistentList()
            data = dict(data)
            # Look if needable table is in input.
            # Default table
            src = self.master
            if 'type' in data and data['type'] in self.db.root:
                src = self.db.root[data['type']]
                del data['type']
            else:
                if 'type' in data:
                    del data['type']
            # Search mechanics

            search_results = set()
            for key in src:
                includes = PersistentMapping()
                decision = True
                # Affected family dog-nail

                for parameter in data:
                    if hasattr(src[key], parameter) and str(src[key].__getattribute__(parameter)).lower().find(data[parameter].lower()) != -1:
                        includes[parameter] = True
                    else:
                        # Plaintext search
                        if parameter and isinstance(src[key], oval_definition):
                            for var in src[key].metadata:
                                if self.variable_search(var, parameter) and self.variable_search(var, data[parameter]):
                                    includes[parameter] = True
                        elif parameter and isinstance(src[key], oval_object):
                            for var in src[key].variables:
                                if self.variable_search(var, parameter) and self.variable_search(var, data[parameter]):
                                    includes[parameter] = True
                        # If nothing found
                        if parameter not in includes:
                            includes[parameter] = False

                for parameter in includes:
                    decision = decision and includes[parameter]
                if decision:
                    search_results.add(src[key].id)

            if search_results:
                search_results = list(search_results)
                search_results.sort(cmp = lambda x,y: cmp(int(x.split(':')[-1]), int(y.split(':')[-1])))
                search_results.reverse()
            else:
                search_results = list()
            
            return search_results
        
        else:
            return None

    def produce_xml(self, filename, id_list = list(), easy = 0):
        if filename:

            output_file = open(filename, "w")
            output_file.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +"\n")
            output_file.write("<oval_definitions ")

            ns = dict()
            ns["xmlns"] = "http://oval.mitre.org/XMLSchema/oval-definitions-5"
            ns["xmlns:oval"] = "http://oval.mitre.org/XMLSchema/oval-common-5"
            ns["xmlns:oval-def"] = "http://oval.mitre.org/XMLSchema/oval-definitions-5"
            ns["xmlns:unix-def"] = "http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"
            ns["xmlns:red-def"] = "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"
            ns["xmlns:xsi"] = "http://www.w3.org/2001/XMLSchema-instance"
            ns["xsi:schemaLocation" ] = "http://oval.mitre.org/XMLSchema/oval-common-5 oval-common-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#unix unix-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#linux linux-definitions-schema.xsd"

            for key in ns.keys():
                output_file.write(key+"="+"\""+ns[key]+"\" ")
            output_file.write(">"+"\n")

            date = datetime.datetime.now()
            timestamp = str(date.strftime("%Y-%m-%dT%H:%M:%S"))+"\n"

            generator = etree.Element("generator")
            etree.SubElement(generator, "oval:product_name").text = "Isox [ptsecurity] builder for " + self.name
            etree.SubElement(generator, "oval:product_version").text = "0.1a"
            etree.SubElement(generator, "oval:schema_version").text = "5.10"
            etree.SubElement(generator, "oval:timestamp").text = timestamp

            output_file.write(self.to_string(generator))
            self._garbager(generator)

            job = list()

            if not id_list:
                job = self.master.keys()
            else:
                id_list = list(set(id_list))
                for id in id_list:
                    job.append(id)
                    job += self.collect_dependencies(id)

            job = list(set(job))
            
            definitions = list()
            tests = list()
            objects = list()
            states = list()
            vars = list()

            for key in job:
                if key in self.master and self.master[key] and isinstance(self.master[key], oval_definition):
                    definitions.append(key)
                elif key in self.master and self.master[key] and isinstance(self.master[key], oval_state):
                    states.append(key)
                elif key in self.master and self.master[key] and isinstance(self.master[key], oval_test):
                    tests.append(key)
                elif key in self.master and self.master[key] and isinstance(self.master[key], oval_variable):
                    vars.append(key)
                elif key in self.master and self.master[key] and isinstance(self.master[key], oval_object):
                    objects.append(key)
                else:
                    #TODO: Make logger entry
                    pass

            output_file.write("<definitions>"+"\n")
            for key in definitions:
                xml = self.xml_me(self.definitions[key], easy)
                output_file.write(self.to_string(xml))
                self._garbager(xml)
                self._fssync(output_file)
            output_file.write("</definitions>"+"\n")

            output_file.write("<tests>"+"\n")
            for key in tests:
                xml = self.xml_me(self.tests[key], easy)
                output_file.write(self.to_string(xml))
                self._garbager(xml)
                self._fssync(output_file)
            output_file.write("</tests>"+"\n")

            output_file.write("<objects>"+"\n")
            for key in objects:
                xml = self.xml_me(self.objects[key], easy)
                output_file.write(self.to_string(xml))
                self._garbager(xml)
                self._fssync(output_file)
            output_file.write("</objects>"+"\n")

            output_file.write("<states>"+"\n")
            for key in states:
                xml = self.xml_me(self.states[key], easy)
                output_file.write(self.to_string(xml))
                self._garbager(xml)
                self._fssync(output_file)
            output_file.write("</states>"+"\n")

            output_file.write("<variables>"+"\n")
            for key in vars:
                xml = self.xml_me(self.variables[key], easy)
                output_file.write(self.to_string(xml))
                self._garbager(xml)
                self._fssync(output_file)
            output_file.write("</variables>"+"\n")

            output_file.write("\n"+"</oval_definitions>")
            output_file.close()
        else:
            raise Exception("Filename for output not listed")

    def indent(self, elem, level=0):
        i = "\n" + level*"  "
        if elem and len(elem):
            if not elem.text or not elem.text.strip():
                elem.text = i + "  "
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
            for elem in elem:
                self.indent(elem, level+1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        else:

            if level and (not elem.tail or not elem.tail.strip()):
                elem.tail = i

    def SplitTagXmlns(self, xmlnstag):
        #return xmlns and simple tag name
        m = re.match(r"^\{(.*?)\}(.*)$",xmlnstag)
        if m is not None:
            return (m.group(1),m.group(2))
        else:
            return ("", xmlnstag)

    @lfu_cache(maxsize=100)
    def to_string(self, element):
        if (element and isinstance(element, oval_object)) or (element and isinstance(element, variable)):
            element = self.xml_me(element)
        if hasattr(element, 'tag') and hasattr(element, 'attrib'):
            if not element.text:
                element.text = ""
            for attribute in element.attrib:
                if not element.attrib[attribute]:
                    element.attrib[attribute] = ""
            new_tree = etree.ElementTree(element)
            self.indent(element)
            string = etree.tostring(element,"UTF-8").replace("<?xml version='1.0' encoding='UTF-8'?>", "").strip()
            del new_tree
            if string and isinstance(string,basestring):
                return string
        else:
            return None

    def _garbager(self, root):
        for element in list(root):
            self._garbager(element)
        root.clear()
        del root
    
    def _fssync(self, file):
        file.flush()
        os.fsync(file.fileno())

    def read_variables(self, etreeElement):
        new_var = variable(self.SplitTagXmlns(etreeElement.tag)[1], etreeElement.text, PersistentMapping(etreeElement.attrib))
        for children in list(etreeElement):
            new_var.variables.append(self.read_variables(children))
        return new_var

    def read_object(self, etreeElement):
        if "id" in etreeElement.attrib.keys() and len(etreeElement.attrib['id'].split(":")) >= 4:
            # Find id and create new object according to it type
            id = etreeElement.attrib['id']
            type = etreeElement.attrib['id'].split(":")[2]
            namespace, tag = self.SplitTagXmlns(etreeElement.tag)
            if type == "def":
                oval_obj = oval_definition()
                oval_obj.id = id
                oval_obj.tag = tag
                oval_obj.namespace = namespace
                if "class" in etreeElement.attrib.keys():
                    oval_obj.defclass = etreeElement.attrib['class']
                if "version" in etreeElement.attrib.keys() and str(etreeElement.attrib['version']).isdigit():
                    oval_obj.version = int(etreeElement.attrib['version'])
                if "comment" in etreeElement.attrib.keys():
                    oval_obj.comment = etreeElement.attrib['comment']
                if "deprecated" in etreeElement.attrib.keys():
                    oval_obj.deprecated = etreeElement.attrib['deprecated']
                # Build internals
                for children in list(etreeElement):
                    if self.SplitTagXmlns(children.tag)[1] == "metadata":
                        for metadata_el in list(children):
                            if self.SplitTagXmlns(metadata_el.tag)[1] == "title":
                                oval_obj.title = metadata_el.text
                            elif self.SplitTagXmlns(metadata_el.tag)[1] == "description":
                                oval_obj.description = metadata_el.text
                            elif self.SplitTagXmlns(metadata_el.tag)[1] == "note":
                                oval_obj.notes.append(self.read_variables(metadata_el))
                            else:
                                oval_obj.metadata.append(self.read_variables(metadata_el))
                    else:
                        oval_obj.criteria.append(self.read_variables(children))
                return oval_obj

            elif type == "obj":
                oval_obj = oval_object()
                oval_obj.id = id
                oval_obj.tag = tag
                oval_obj.namespace = namespace
                if "version" in etreeElement.attrib.keys() and str(etreeElement.attrib['version']).isdigit():
                    oval_obj.version = int(etreeElement.attrib['version'])
                if "comment" in etreeElement.attrib.keys():
                    oval_obj.comment = etreeElement.attrib['comment']
                if "deprecated" in etreeElement.attrib.keys():
                    oval_obj.deprecated = etreeElement.attrib['deprecated']
                # Build internals
                for children in list(etreeElement):
                    if self.SplitTagXmlns(children.tag)[1] == "note":
                        oval_obj.notes.append(self.read_variables(children))
                    else:
                        oval_obj.variables.append(self.read_variables(children))
                return oval_obj

            elif type == "tst":
                oval_obj = oval_test()
                oval_obj.id = id
                oval_obj.tag = tag
                oval_obj.namespace = namespace
                if "version" in etreeElement.attrib.keys() and str(etreeElement.attrib['version']).isdigit():
                    oval_obj.version = int(etreeElement.attrib['version'])
                if "comment" in etreeElement.attrib.keys():
                    oval_obj.comment = etreeElement.attrib['comment']
                if "deprecated" in etreeElement.attrib.keys():
                    oval_obj.deprecated = etreeElement.attrib['deprecated']
                if "check" in etreeElement.attrib.keys():
                    oval_obj.check = etreeElement.attrib['check']
                if "check_existance" in etreeElement.attrib.keys():
                    oval_obj.check_existance = etreeElement.attrib['check_existance']
                if "state_operator" in etreeElement.attrib.keys():
                    oval_obj.state_operator = etreeElement.attrib['state_operator']
                # Build internals
                for children in list(etreeElement):
                    if self.SplitTagXmlns(children.tag)[1] == "note":
                        oval_obj.notes.append(self.read_variables(children))
                    else:
                        oval_obj.variables.append(self.read_variables(children))
                return oval_obj

            elif type == "ste":
                oval_obj = oval_state()
                oval_obj.id = id
                oval_obj.tag = tag
                oval_obj.namespace = namespace
                if "version" in etreeElement.attrib.keys() and str(etreeElement.attrib['version']).isdigit():
                    oval_obj.version = int(etreeElement.attrib['version'])
                if "comment" in etreeElement.attrib.keys():
                    oval_obj.comment = etreeElement.attrib['comment']
                if "deprecated" in etreeElement.attrib.keys():
                    oval_obj.deprecated = etreeElement.attrib['deprecated']
                if "operator" in etreeElement.attrib.keys():
                    oval_obj.operator = etreeElement.attrib['operator']
                # Build internals
                for children in list(etreeElement):
                    if self.SplitTagXmlns(children.tag)[1] == "note":
                        oval_obj.notes.append(self.read_variables(children))
                    else:
                        oval_obj.variables.append(self.read_variables(children))
                return oval_obj

            elif type == "var":
                oval_obj = oval_variable()
                oval_obj.id = id
                oval_obj.tag = tag
                oval_obj.namespace = namespace
                if "version" in etreeElement.attrib.keys() and str(etreeElement.attrib['version']).isdigit():
                    oval_obj.version = int(etreeElement.attrib['version'])
                if "comment" in etreeElement.attrib.keys():
                    oval_obj.comment = etreeElement.attrib['comment']
                if "deprecated" in etreeElement.attrib.keys():
                    oval_obj.deprecated = etreeElement.attrib['deprecated']
                if "datatype" in etreeElement.attrib.keys():
                    oval_obj.datatype = etreeElement.attrib['datatype']
                # Build internals
                for children in list(etreeElement):
                    if self.SplitTagXmlns(children.tag)[1] == "note":
                        oval_obj.notes.append(self.read_variables(children))
                    else:
                        oval_obj.variables.append(self.read_variables(children))
                return oval_obj

            else:
                raise Exception("Error occured: Looks like not OVAL XML")
        else:
            raise Exception("Error occured: Wrong Etree element input")

    def parse_oval_xml(self, filename):
        if filename and isinstance(filename, basestring):
            xml_structured = oval_mapper(filename)

            for key in xml_structured.definitions.keys():
                self.put(self.read_object(xml_structured.definitions[key]))

            for key in xml_structured.objects.keys():
                self.put(self.read_object(xml_structured.objects[key]))

            for key in xml_structured.tests.keys():
                self.put(self.read_object(xml_structured.tests[key]))

            for key in xml_structured.states.keys():
                self.put(self.read_object(xml_structured.states[key]))

            for key in xml_structured.vars.keys():
                self.put(self.read_object(xml_structured.vars[key]))
        else:
            raise Exception("Must provide correct filename")

    def get_variable_deps(self, var, pattern):
        if var and isinstance(var, variable) and pattern and isinstance(pattern, basestring):
            result = list()
            for value in var.attributes.values():
                if re.findall(pattern, value):
                    result += re.findall(pattern, value)
            for child in var.variables:
                result += self.get_variable_deps(child, pattern)
            return list(set(result))

    def collect_dependencies(self, id, pattern = "(oval:.+?:.+?:\d+)"):
        if id and isinstance(id, basestring) and id in self.master and self.master[id]:
            if isinstance(self.master[id], oval_definition):
                result = list()
                for var in self.master[id].criteria:
                    result += self.get_variable_deps(var, pattern)
                for dep_id in result:
                    result += self.collect_dependencies(dep_id, pattern)
                return list(set(result))
            elif isinstance(self.master[id], oval_object):
                result = list()
                for var in self.master[id].variables:
                    result += self.get_variable_deps(var, pattern)
                for dep_id in result:
                    result += self.collect_dependencies(dep_id, pattern)
                return list(set(result))
            else:
                return list()
        else:
            return list()

    @lfu_cache(maxsize=50)
    def string_xml(self, id_list = frozenset(), easy = 0):


        output = str()
        output += "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +"\n"
        output += "<oval_definitions "

        ns = dict()
        ns["xmlns"] = "http://oval.mitre.org/XMLSchema/oval-definitions-5"
        ns["xmlns:oval"] = "http://oval.mitre.org/XMLSchema/oval-common-5"
        ns["xmlns:oval-def"] = "http://oval.mitre.org/XMLSchema/oval-definitions-5"
        ns["xmlns:unix-def"] = "http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"
        ns["xmlns:red-def"] = "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"
        ns["xmlns:xsi"] = "http://www.w3.org/2001/XMLSchema-instance"
        ns["xsi:schemaLocation"] = "http://oval.mitre.org/XMLSchema/oval-common-5 oval-common-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#unix unix-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#linux linux-definitions-schema.xsd"

        for key in ns.keys():
            output += key+"="+"\""+ns[key]+"\" "
        output += ">"+"\n"

        date = datetime.datetime.now()
        timestamp = str(date.strftime("%Y-%m-%dT%H:%M:%S"))+"\n"

        generator = etree.Element("generator")
        etree.SubElement(generator, "oval:product_name").text = "Isox [ptsecurity] builder for " + self.name
        etree.SubElement(generator, "oval:product_version").text = self.product_version
        etree.SubElement(generator, "oval:schema_version").text = self.schema_version
        etree.SubElement(generator, "oval:timestamp").text = timestamp

        output += self.to_string(generator)
        self._garbager(generator)

        job = list()

        if not id_list:
            job = self.master.keys()
        else:
            id_list = list(set(id_list))
            for id in id_list:
                job.append(id)
                job += self.collect_dependencies(id)

        job = list(set(job))

        definitions = list()
        tests = list()
        objects = list()
        states = list()
        vars = list()

        for key in job:
            if key in self.master and self.master[key] and isinstance(self.master[key], oval_definition):
                definitions.append(key)
            elif key in self.master and self.master[key] and isinstance(self.master[key], oval_state):
                states.append(key)
            elif key in self.master and self.master[key] and isinstance(self.master[key], oval_test):
                tests.append(key)
            elif key in self.master and self.master[key] and isinstance(self.master[key], oval_variable):
                vars.append(key)
            elif key in self.master and self.master[key] and isinstance(self.master[key], oval_object):
                objects.append(key)
            else:
                #TODO: Make logger entry
                pass

        output += "<definitions>"+"\n"
        for key in definitions:
            xml = self.xml_me(self.definitions[key], easy)
            output += self.to_string(xml)
            self._garbager(xml)
        output += "</definitions>"+"\n"

        output += "<tests>"+"\n"
        for key in tests:
            xml = self.xml_me(self.tests[key], easy)
            output += self.to_string(xml)
            self._garbager(xml)
        output += "</tests>"+"\n"

        output += "<objects>"+"\n"
        for key in objects:
            xml = self.xml_me(self.objects[key], easy)
            output += self.to_string(xml)
            self._garbager(xml)
        output += "</objects>"+"\n"

        output += "<states>"+"\n"
        for key in states:
            xml = self.xml_me(self.states[key], easy)
            output += self.to_string(xml)
            self._garbager(xml)
        output += "</states>"+"\n"
        
        output += "<variables>"+"\n"
        for key in vars:
            xml = self.xml_me(self.variables[key], easy)
            output += self.to_string(xml)
            self._garbager(xml)
        output += "</variables>"+"\n"

        output += "\n"+"</oval_definitions>"
        return output

            